# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json, redis, pickle
from scrapy.conf import settings
from scrapy.exceptions import DropItem


class JdbookPipeline(object):
    # def process_item(self, item, spider):
    #     with open("data.txt", "a") as f:
    #         json.dump(item, f, ensure_ascii=False, indent=2)
    #         f.write("\n")
    #     return item
    def __init__(self):
        # redis_pool = redis.ConnectionPool(host='bw.ztm.me', port=6379, db=0, password='Schenker123')
        redis_pool = redis.ConnectionPool(host='127.0.0.1', port=6379, db=0)
        self.server = redis.Redis(connection_pool=redis_pool)
        print('start job')
        self.f = open('jd_book.json', 'w')
        # self.f.write('[')

    def process_item(self, item, spider):
        valid = True
        for data in item:
            if not data:
                valid = False
                raise DropItem("Missing {0}!".format(data))
        if valid:
            try:
                # This returns the number of values added, zero if already exists.
                key = item['sku']
                item_name = item['name']
                # if self.server.get(key) is not None:
                if not self.server.hexists(name='jdbook', key=key) and not item_name.find('预定') > 0:
                    self.server.hset(name='jdbook', key=key, value=item_name)
                    # self.server.set(key, item_name)
                    json.dump(item, self.f, ensure_ascii=False)
                    self.f.write('\n')
                    # with open("jd_book.json", "a") as f:
                    #      json.dump(item, f, ensure_ascii=False)
                    #      f.write("\n")
                    print("Saved {} | {} | {}".format(item['top'], item['sub'], item['name']))
            except Exception as err:
                print(err)
                # raise DropItem("Duplicated Item: {}".format(item['name']))
        return item
    def close_spider(self, spider):
        # self.f.write(']')
        self.f.close()
        print('job done')
